#ifndef __TOKENIZER_H_
#define __TOKENIZER_H_

#include <string>
#include <iostream>
#include <vector>
#include <fstream>
#include "util.h"

enum TokenType {
  WHITESPACE_TOK = 0,
  WORD_TOK,
  NUMBER_TOK,
  PUNCTUATION_TOK,
  ESCAPE_TOK,
  TAG_TOK,
  END_TOK,
  ERROR_TOK,
};

// Tokens always point at the original thing in [start_, end_)
struct Token {
Token(TokenType type, char * start, char * end) 
: type_(type), start_(start), end_(end) {}
Token() : type_(ERROR_TOK), start_(NULL), end_(NULL) {}
  string String() { return string(start_, end_ - start_); }

  TokenType type_;
  char * start_;
  char * end_;
};

struct Tokenizer {

  void Init(char * start, char * end) {
    CHECK(*end == '\0');
    start_ = current_ = start;
    end_ = end;
    count_ = 0;
  }

  void InitFromFile(string fn);
  void InitFromBlock(Memblock m);

  // Easy usage: call Init, then GetNextToken, returns NULL on end
  Token * GetNextToken() {
    count_++;
    if (count_ % 1000000 == 0) cout << "Tokenizing #tokens = " << count_ << endl;
    if (AtEnd()) return NULL;
    NextToken();
    if (token_.type_ == END_TOK) return NULL;
    return &token_;
  }

  // Fills token_. Usage: call NextToken, GetToken, check AtEnd
  virtual void NextToken() = 0;
  bool AtEnd() { return (current_ == end_); }
  
  void Reset() {
    current_ = start_;
    count_ = 0;
  }

  // data
  char * start_;
  char * end_;
  char * current_;
  Token token_;
  uint64 count_;
};

struct XMLTokenizer : public Tokenizer {
  virtual void NextToken();
};

int tokenize_main();


#endif
